const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  let totalBlogs = [];
  const MAX_CNT = 300;
  const SLEEP_TIME = 200;

  const fileName = `tuicool.json`;

  const browser = await puppeteer.launch({headless: true, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});
  page.on('console', msg => {
    console.log(msg.text());
  });

  page.on('pageerror', msg => {
    console.log(msg);
  });

  let pageSize = 500;

  let url = 'https://www.tuicool.com/mags';
  await page.goto(url);
  await page.waitFor(1000);

  let newUrl = await page.$eval('body > div.container-fluid > div.mag.zhoukan_mag > div > div:nth-child(1) > a', ele => ele.href);

  await page.goto(newUrl);
  await page.waitFor(1000);
  let blogs = null;

  // 后端技术
  blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span8 > div > ol:nth-child(11) > li', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('h4 > a').title;
      let content = ele.querySelector('h4 > a').href;
      let source = ele.querySelector('h4 > span').innerText;
      tmpBlogs.push({
        title: title + source,
        content: content,
        tab: 'tuicool',
        sub_tab: 'frontend'
      });
    });
    return tmpBlogs;
  });
  console.log(blogs);
  console.log('后端技术over--------------');
  totalBlogs = totalBlogs.concat(blogs);


  // 编程语言
  blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span8 > div > ol:nth-child(7) > li', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('h4 > a').title;
      let content = ele.querySelector('h4 > a').href;
      let source = ele.querySelector('h4 > span').innerText;
      tmpBlogs.push({
        title: title + source,
        content: content,
        tab: 'tuicool',
        sub_tab: 'bianchengyuyan'
      });
    });
    return tmpBlogs;
  });
  console.log(blogs);
  console.log('编程语言over--------------');
  totalBlogs = totalBlogs.concat(blogs);

  // 数据存储
  blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span8 > div > ol:nth-child(9) > li', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('h4 > a').title;
      let content = ele.querySelector('h4 > a').href;
      let source = ele.querySelector('h4 > span').innerText;
      tmpBlogs.push({
        title: title + source,
        content: content,
        tab: 'tuicool',
        sub_tab: 'shujucunchu'
      });
    });
    return tmpBlogs;
  });
  console.log(blogs);
  console.log('数据存储over--------------');
  totalBlogs = totalBlogs.concat(blogs);


  // 前端开发
  blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span8 > div > ol:nth-child(3) > li', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('h4 > a').title;
      let content = ele.querySelector('h4 > a').href;
      let source = ele.querySelector('h4 > span').innerText;
      tmpBlogs.push({
        title: title + source,
        content: content,
        tab: 'tuicool',
        sub_tab: 'frontend'
      });
    });
    return tmpBlogs;
  });
  console.log(blogs);
  console.log('前端开发over--------------');
  totalBlogs = totalBlogs.concat(blogs);


  // 数据科学
  blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span8 > div > ol:nth-child(13) > li', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('h4 > a').title;
      let content = ele.querySelector('h4 > a').href;
      let source = ele.querySelector('h4 > span').innerText;
      tmpBlogs.push({
        title: title + source,
        content: content,
        tab: 'tuicool',
        sub_tab: 'shujukexue'
      });
    });
    return tmpBlogs;
  });
  console.log(blogs);
  console.log('数据科学over--------------');
  totalBlogs = totalBlogs.concat(blogs);


  // 安全技术
  blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span8 > div > ol:nth-child(15) > li', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('h4 > a').title;
      let content = ele.querySelector('h4 > a').href;
      let source = ele.querySelector('h4 > span').innerText;
      tmpBlogs.push({
        title: title + source,
        content: content,
        tab: 'tuicool',
        sub_tab: 'anquanjishu'
      });
    });
    return tmpBlogs;
  });
  console.log(blogs);
  console.log('安全技术over--------------');
  totalBlogs = totalBlogs.concat(blogs);


  // 技术纵横
  blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span8 > div > ol:nth-child(17) > li', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('h4 > a').title;
      let content = ele.querySelector('h4 > a').href;
      let source = ele.querySelector('h4 > span').innerText;
      tmpBlogs.push({
        title: title + source,
        content: content,
        tab: 'tuicool',
        sub_tab: 'jishuzongheng'
      });
    });
    return tmpBlogs;
  });
  console.log(blogs);
  console.log('技术纵横over--------------');
  totalBlogs = totalBlogs.concat(blogs);


  // 图书推荐
  blogs = await page.$$eval('body > div.container-fluid > div.row-fluid > div.span8 > div > ol:nth-child(19) > li', async (eles) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('h4 > a').title;
      let content = ele.querySelector('h4 > a').href;
      tmpBlogs.push({
        title: title,
        content: content,
        tab: 'tuicool',
        sub_tab: 'tushutuijian'
      });
    });
    return tmpBlogs;
  });
  console.log(blogs);
  console.log('图书推荐over--------------');
  totalBlogs = totalBlogs.concat(blogs);


  let writerStream = fs.createWriteStream(`crawler/crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

  console.log(errLogs);

  // browser.close();
};


main();


